},
        "instance_cropping": {
            "center_on_part": "abdomen",
            "crop_size": 576,
            "crop_size_detection_padding": 16
        }
    },
    "model": {
        "backbone": {
            "leap": null,
            "unet": {
                "stem_stride": null,
                "max_stride": 32,
                "output_stride": 2,
                "filters": 24,
                "filters_rate": 1.5,
                "middle_block": true,
                "up_interpolate": true,
                "stacks": 1
            },
            "hourglass": null,
            "resnet": null,
            "pretrained_encoder": null
        },
        "heads": {
            "single_instance": null,
            "centroid": {
                "anchor_part": "abdomen",
                "sigma": 2.5,
                "output_stride": 2,
                "loss_weight": 1.0,
                "offset_refinement": false
            },
            "centered_instance": null,
            "multi_instance": null,
            "multi_class_bottomup": null,
            "multi_class_topdown": null
        },
        "base_checkpoint": null
    },
    "optimization": {
        "preload_data": true,
        "augmentation_config": {
            "rotate": true,
            "rotation_min_angle": -180.0,
            "rotation_max_angle": 180.0,
            "translate": false,
            "translate_min": -5,
            "translate_max": 5,
            "scale": false,
            "scale_min": 0.9,
            "scale_max": 1.1,
            "uniform_noise": false,
            "uniform_noise_min_val": 0.0,
            "uniform_noise_max_val": 10.0,
            "gaussian_noise": false,
            "gaussian_noise_mean": 5.0,
            "gaussian_noise_stddev": 1.0,
            "contrast": false,
            "contrast_min_gamma": 0.5,
            "contrast_max_gamma": 2.0,
            "brightness": false,
            "brightness_min_val": 0.0,
            "brightness_max_val": 10.0,
            "random_crop": false,
            "random_crop_height": 256,
            "random_crop_width": 256,
            "random_flip": false,
            "flip_horizontal": true
        },
        "online_shuffling": true,
        "shuffle_buffer_size": 128,
        "prefetch": true,
        "batch_size": 4,
        "batches_per_epoch": 200,
        "min_batches_per_epoch": 200,
        "val_batches_per_epoch": 10,
        "min_val_batches_per_epoch": 10,
        "epochs": 2,
        "optimizer": "adam",
        "initial_learning_rate": 0.0001,
        "learning_rate_schedule": {
            "reduce_on_plateau": true,
            "reduction_factor": 0.5,
            "plateau_min_delta": 1e-06,
            "plateau_patience": 5,
            "plateau_cooldown": 3,
            "min_learning_rate": 1e-08
        },
        "hard_keypoint_mining": {
            "online_mining": false,
            "hard_to_easy_ratio": 2.0,
            "min_hard_keypoints": 2,
            "max_hard_keypoints": null,
            "loss_scale": 5.0
        },
        "early_stopping": {
            "stop_training_on_plateau": true,
            "plateau_min_delta": 1e-08,
            "plateau_patience": 20
        }
    },
    "outputs": {
        "save_outputs": true,
        "run_name": "241009_114324.centroid.n=3",
        "run_name_prefix": "",
        "run_name_suffix": "",
        "runs_folder": "D:/social-leap-estimates-animal-poses/datasets/drosophila-melanogaster-courtship/drosophila-melanogaster-courtship\\models",
        "tags": [
            ""
        ],
        "save_visualizations": true,
        "keep_viz_images": false,
        "zip_outputs": false,
        "log_to_csv": true,
        "checkpointing": {
            "initial_model": false,
            "best_model": true,
            "every_epoch": false,
            "latest_model": false,
            "final_model": false
        },
        "tensorboard": {
            "write_logs": false,
            "loss_frequency": "epoch",
            "architecture_graph": false,
            "profile_graph": false,
            "visualizations": true
        },
        "zmq": {
            "subscribe_to_controller": true,
            "controller_address": "tcp://127.0.0.1:8998",
            "controller_polling_timeout": 10,
            "publish_updates": true,
            "publish_address": "tcp://127.0.0.1:9001"
        }
    },
    "name": "",
    "description": "",
    "sleap_version": "1.4.1a2",
    "filename": "C:\\Users\\TalmoLab\\AppData\\Local\\Temp\\tmp4fv5srb5\\241009_114325_training_job.json"
}
INFO:sleap.nn.training:
INFO:sleap.nn.training:Auto-selected GPU 0 with 22982 MiB of free memory.
INFO:sleap.nn.training:Using GPU 0 for acceleration.
INFO:sleap.nn.training:Disabled GPU memory pre-allocation.
INFO:sleap.nn.training:System:
GPUs: 1/1 available
  Device: /physical_device:GPU:0
         Available: True
       Initialized: False
     Memory growth: True
INFO:sleap.nn.training:
INFO:sleap.nn.training:Initializing trainer...
INFO:sleap.nn.training:Loading training labels from: D:/social-leap-estimates-animal-poses/datasets/drosophila-melanogaster-courtship/drosophila-melanogaster-courtship/labels.v001.slp
INFO:sleap.nn.training:Creating training and validation splits from validation fraction: 0.1
INFO:sleap.nn.training:  Splits: Training = 2 / Validation = 1.
INFO:sleap.nn.training:Setting up for training...
INFO:sleap.nn.training:Setting up pipeline builders...
INFO:sleap.nn.training:Setting up model...
INFO:sleap.nn.training:Building test pipeline...
2024-10-09 11:43:37.689755: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX AVX2
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
2024-10-09 11:43:38.141710: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1532] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 21631 MB memory:  -> device: 0, name: NVIDIA RTX A5000, pci bus id: 0000:01:00.0, compute capability: 8.6
INFO:sleap.nn.training:Loaded test example. [1.944s]
INFO:sleap.nn.training:  Input shape: (1024, 1024, 1)
INFO:sleap.nn.training:Created Keras model.
INFO:sleap.nn.training:  Backbone: UNet(stacks=1, filters=24, filters_rate=1.5, kernel_size=3, stem_kernel_size=7, convs_per_block=2, stem_blocks=0, down_blocks=5, middle_block=True, up_blocks=4, up_interpolate=True, block_contraction=False)
INFO:sleap.nn.training:  Max stride: 32
INFO:sleap.nn.training:  Parameters: 1,685,625
INFO:sleap.nn.training:  Heads:
INFO:sleap.nn.training:    [0] = CentroidConfmapsHead(anchor_part='abdomen', sigma=2.5, output_stride=2, loss_weight=1.0)
INFO:sleap.nn.training:  Outputs:
INFO:sleap.nn.training:    [0] = KerasTensor(type_spec=TensorSpec(shape=(None, 512, 512, 1), dtype=tf.float32, name=None), name='CentroidConfmapsHead/BiasAdd:0', description="created by layer 'CentroidConfmapsHead'")
INFO:sleap.nn.training:Training from scratch
INFO:sleap.nn.training:Setting up data pipelines...
INFO:sleap.nn.training:Training set: n = 2
INFO:sleap.nn.training:Validation set: n = 1
INFO:sleap.nn.training:Setting up optimization...
INFO:sleap.nn.training:  Learning rate schedule: LearningRateScheduleConfig(reduce_on_plateau=True, reduction_factor=0.5, plateau_min_delta=1e-06, plateau_patience=5, plateau_cooldown=3, min_learning_rate=1e-08)
INFO:sleap.nn.training:  Early stopping: EarlyStoppingConfig(stop_training_on_plateau=True, plateau_min_delta=1e-08, plateau_patience=20)
INFO:sleap.nn.training:Setting up outputs...
INFO:sleap.nn.callbacks:Training controller subscribed to: tcp://127.0.0.1:8998 (topic: )
INFO:sleap.nn.training:  ZMQ controller subcribed to: tcp://127.0.0.1:8998
INFO:sleap.nn.callbacks:Progress reporter publishing on: tcp://127.0.0.1:9001 for: not_set
INFO:sleap.nn.training:  ZMQ progress reporter publish on: tcp://127.0.0.1:9001
INFO:sleap.nn.training:Created run path: D:/social-leap-estimates-animal-poses/datasets/drosophila-melanogaster-courtship/drosophila-melanogaster-courtship\models\241009_114324.centroid.n=3
INFO:sleap.nn.training:Setting up visualization...
2024-10-09 11:43:41.557756: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: -34 } dim { size: -35 } dim { size: -36 } dim { size: 1 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "GPU" vendor: "NVIDIA" model: "NVIDIA RTX A5000" frequency: 1695 num_cores: 64 environment { key: "architecture" value: "8.6" } environment { key: "cuda" value: "11020" } environment { key: "cudnn" value: "8100" } num_registers: 65536 l1_cache_size: 24576 l2_cache_size: 6291456 shared_memory_size_per_multiprocessor: 102400 memory_size: 22681944064 bandwidth: 768096000 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: -37 } dim { size: -38 } dim { size: 1 } } }
2024-10-09 11:43:42.437965: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: -34 } dim { size: -35 } dim { size: -36 } dim { size: 1 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "GPU" vendor: "NVIDIA" model: "NVIDIA RTX A5000" frequency: 1695 num_cores: 64 environment { key: "architecture" value: "8.6" } environment { key: "cuda" value: "11020" } environment { key: "cudnn" value: "8100" } num_registers: 65536 l1_cache_size: 24576 l2_cache_size: 6291456 shared_memory_size_per_multiprocessor: 102400 memory_size: 22681944064 bandwidth: 768096000 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: -37 } dim { size: -38 } dim { size: 1 } } }
INFO:sleap.nn.training:Finished trainer set up. [4.8s]
INFO:sleap.nn.training:Creating tf.data.Datasets for training data generation...
INFO:sleap.nn.training:Finished creating training datasets. [1.7s]
INFO:sleap.nn.training:Starting training loop...
Epoch 1/2
2024-10-09 11:43:47.494736: I tensorflow/stream_executor/cuda/cuda_dnn.cc:384] Loaded cuDNN version 8201
WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.0106s vs `on_train_batch_end` time: 0.1389s). Check your callbacks.
2024-10-09 11:44:42.094369: W tensorflow/core/common_runtime/bfc_allocator.cc:290] Allocator (GPU_0_bfc) ran out of memory trying to allocate 3.28GiB with freed_by_count=0. The caller indicates that this is not a failure, but this may mean that there could be performance gains if more memory were available.
2024-10-09 11:44:42.095361: W tensorflow/core/common_runtime/bfc_allocator.cc:290] Allocator (GPU_0_bfc) ran out of memory trying to allocate 3.28GiB with freed_by_count=0. The caller indicates that this is not a failure, but this may mean that there could be performance gains if more memory were available.
2024-10-09 11:44:42.096274: W tensorflow/core/common_runtime/bfc_allocator.cc:290] Allocator (GPU_0_bfc) ran out of memory trying to allocate 3.28GiB with freed_by_count=0. The caller indicates that this is not a failure, but this may mean that there could be performance gains if more memory were available.
2024-10-09 11:44:42.097218: W tensorflow/core/common_runtime/bfc_allocator.cc:290] Allocator (GPU_0_bfc) ran out of memory trying to allocate 3.28GiB with freed_by_count=0. The caller indicates that this is not a failure, but this may mean that there could be performance gains if more memory were available.
2024-10-09 11:44:45.250313: I tensorflow/stream_executor/cuda/cuda_blas.cc:1786] TensorFloat-32 will be used for the matrix multiplication. This will only be logged once.
200/200 - 62s - loss: 1.3333e-04 - val_loss: 6.3918e-05 - lr: 1.0000e-04 - 62s/epoch - 312ms/step
Epoch 2/2
Polling: D:/social-leap-estimates-animal-poses/datasets/drosophila-melanogaster-courtship/drosophila-melanogaster-courtship\models\241009_114324.centroid.n=3\viz\validation.*.png
200/200 - 40s - loss: 4.7547e-05 - val_loss: 7.7009e-05 - lr: 1.0000e-04 - 40s/epoch - 202ms/step
Polling: D:/social-leap-estimates-animal-poses/datasets/drosophila-melanogaster-courtship/drosophila-melanogaster-courtship\models\241009_114324.centroid.n=3\viz\validation.*.png
INFO:sleap.nn.training:Finished training loop. [1.7 min]
INFO:sleap.nn.training:Deleting visualization directory: D:/social-leap-estimates-animal-poses/datasets/drosophila-melanogaster-courtship/drosophila-melanogaster-courtship\models\241009_114324.centroid.n=3\viz
INFO:sleap.nn.training:Saving evaluation metrics to model folder...
Predicting... ----------------------------------------   0% ETA: -:--:-- ?2024-10-09 11:45:30.200022: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: -65 } dim { size: -66 } dim { size: -67 } dim { size: 1 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "GPU" vendor: "NVIDIA" model: "NVIDIA RTX A5000" frequency: 1695 num_cores: 64 environment { key: "architecture" value: "8.6" } environment { key: "cuda" value: "11020" } environment { key: "cudnn" value: "8100" } num_registers: 65536 l1_cache_size: 24576 l2_cache_size: 6291456 shared_memory_size_per_multiprocessor: 102400 memory_size: 22681944064 bandwidth: 768096000 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: -68 } dim { size: -69 } dim { size: 1 } } }
2024-10-09 11:45:30.201453: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_UINT8 } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_UINT8 shape { dim { size: 2 } dim { size: 1024 } dim { size: 1024 } dim { size: 1 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "103" frequency: 3600 num_cores: 16 environment { key: "cpu_instruction_set" value: "SSE, SSE2" } environment { key: "eigen" value: "3.4.90" } l1_cache_size: 49152 l2_cache_size: 524288 l3_cache_size: 16777216 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: -77 } dim { size: -78 } dim { size: 1 } } }
Predicting... ---------------------------------------- 100% ETA: 0:00:00 ?
C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\sleap\nn\evals.py:539: RuntimeWarning: Mean of empty slice
  "dist.avg": np.nanmean(dists),
C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\sleap\nn\evals.py:572: RuntimeWarning: Mean of empty slice.
  mPCK = mPCK_parts.mean()
C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\numpy\core\_methods.py:129: RuntimeWarning: invalid value encountered in scalar divide
  ret = ret.dtype.type(ret / rcount)
C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\sleap\nn\evals.py:666: RuntimeWarning: Mean of empty slice.
  pair_pck = metrics["pck.pcks"].mean(axis=-1).mean(axis=-1)
C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\numpy\core\_methods.py:121: RuntimeWarning: invalid value encountered in divide

  ret = um.true_divide(
C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\sleap\nn\evals.py:668: RuntimeWarning: Mean of empty slice.
  metrics["oks.mOKS"] = pair_oks.mean()
WARNING:sleap.nn.evals:Failed to compute metrics.
INFO:sleap.nn.evals:Saved predictions: D:/social-leap-estimates-animal-poses/datasets/drosophila-melanogaster-courtship/drosophila-melanogaster-courtship\models\241009_114324.centroid.n=3\labels_pr.train.slp
Predicting... ----------------------------------------   0% ETA: -:--:-- ?2024-10-09 11:45:33.354849: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: -65 } dim { size: -66 } dim { size: -67 } dim { size: 1 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "GPU" vendor: "NVIDIA" model: "NVIDIA RTX A5000" frequency: 1695 num_cores: 64 environment { key: "architecture" value: "8.6" } environment { key: "cuda" value: "11020" } environment { key: "cudnn" value: "8100" } num_registers: 65536 l1_cache_size: 24576 l2_cache_size: 6291456 shared_memory_size_per_multiprocessor: 102400 memory_size: 22681944064 bandwidth: 768096000 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: -68 } dim { size: -69 } dim { size: 1 } } }
2024-10-09 11:45:33.356694: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_UINT8 } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_UINT8 shape { dim { size: 1 } dim { size: 1024 } dim { size: 1024 } dim { size: 1 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "103" frequency: 3600 num_cores: 16 environment { key: "cpu_instruction_set" value: "SSE, SSE2" } environment { key: "eigen" value: "3.4.90" } l1_cache_size: 49152 l2_cache_size: 524288 l3_cache_size: 16777216 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: -77 } dim { size: -78 } dim { size: 1 } } }
Predicting... ---------------------------------------- 100% ETA: 0:00:00 ?
WARNING:sleap.nn.evals:Failed to compute metrics.
INFO:sleap.nn.evals:Saved predictions: D:/social-leap-estimates-animal-poses/datasets/drosophila-melanogaster-courtship/drosophila-melanogaster-courtship\models\241009_114324.centroid.n=3\labels_pr.val.slp
INFO:sleap.nn.callbacks:Closing the reporter controller/context.
INFO:sleap.nn.callbacks:Closing the training controller socket/context.
Run Path: D:/social-leap-estimates-animal-poses/datasets/drosophila-melanogaster-courtship/drosophila-melanogaster-courtship\models\241009_114324.centroid.n=3
Finished training centroid.
Resetting monitor window.
Polling: D:/social-leap-estimates-animal-poses/datasets/drosophila-melanogaster-courtship/drosophila-melanogaster-courtship\models\241009_114536.centered_instance.n=3\viz\validation.*.png
Start training centered_instance...
['sleap-train', 'C:\\Users\\TalmoLab\\AppData\\Local\\Temp\\tmp1y6bl0lp\\241009_114536_training_job.json', 'D:/social-leap-estimates-animal-poses/datasets/drosophila-melanogaster-courtship/drosophila-melanogaster-courtship/labels.v001.slp', '--zmq', '--controller_port', '8998', '--publish_port', '9001', '--save_viz']
C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\albumentations\__init__.py:13: UserWarning: A new version of Albumentations is available: 1.4.18 (you have 1.4.15). Upgrade using: pip install -U albumentations. To disable automatic update checks, set the environment variable NO_ALBUMENTATIONS_UPDATE to 1.
  check_for_updates()
INFO:sleap.nn.training:Versions:
SLEAP: 1.4.1a3
TensorFlow: 2.9.2
Numpy: 1.26.4
Python: 3.10.15
OS: Windows-10-10.0.19044-SP0
INFO:sleap.nn.training:Training labels file: D:/social-leap-estimates-animal-poses/datasets/drosophila-melanogaster-courtship/drosophila-melanogaster-courtship/labels.v001.slp
INFO:sleap.nn.training:Training profile: C:\Users\TalmoLab\AppData\Local\Temp\tmp1y6bl0lp\241009_114536_training_job.json
INFO:sleap.nn.training:
INFO:sleap.nn.training:Arguments:
INFO:sleap.nn.training:{
    "training_job_path": "C:\\Users\\TalmoLab\\AppData\\Local\\Temp\\tmp1y6bl0lp\\241009_114536_training_job.json",
    "labels_path": "D:/social-leap-estimates-animal-poses/datasets/drosophila-melanogaster-courtship/drosophila-melanogaster-courtship/labels.v001.slp",
    "video_paths": [
        ""
    ],
    "val_labels": null,
    "test_labels": null,
    "base_checkpoint": null,
    "tensorboard": false,
    "save_viz": true,
    "keep_viz": false,
    "zmq": true,
    "publish_port": 9001,
    "controller_port": 8998,
    "run_name": "",
    "prefix": "",
    "suffix": "",
    "cpu": false,
    "first_gpu": false,
    "last_gpu": false,
    "gpu": "auto"
}
INFO:sleap.nn.training:
INFO:sleap.nn.training:Training job:
INFO:sleap.nn.training:{
    "data": {
        "labels": {
            "training_labels": "D:\\social-leap-estimates-animal-poses\\datasets\\drosophila-melanogaster-courtship\\drosophila-melanogaster-courtship\\courtship_labels.slp",
            "validation_labels": null,
            "validation_fraction": 0.1,
            "test_labels": null,
            "split_by_inds": false,
            "training_inds": [
                38,
                35,
                63,
                11,
                13,
                42,
                78,
                25,
                61,
                57,
                79,
                24,
                85,
                12,
                89,
                64,
                18,
                96,
                32,
                72,
                26,
                20,
                46,
                68,
                84,
                6,
                59,
                73,
                17,
                75,
                29,
                66,
                56,
                7,
                9,
                77,
                31,
                41,
                80,
                94,
                76,
                27,
                15,
                60,
                39,
                45,
                49,
                69,
                92,
                65,
                55,
                34,
                48,
                16,
                33,
                14,
                2,
                8,
                44,
                28,
                47,
                21,
                54,
                87,
                3,
                37,
                99,
                98,
                58,
                4,
                10,
                0,
                95,
                91,
                50,
                22,
                67,
                74,
                40,
                82,
                62,
                19,
                86,
                36,
                88,
                51,
                30,
                71,
                23,
                83,
                52
            ],
            "validation_inds": [
                53,
                43,
                100,
                90,
                1,
                70,
                97,
                5,
                93,
                81
            ],
            "test_inds": null,
            "search_path_hints": [
                "",
                "",
                "",
                "",
                "",
                "",
                "",
                "",
                "",
                "",
                "",
                "",
                "",
                "",
                "",
                "",
                "",
                "",
                ""
            ],
            "skeletons": []
        },
        "preprocessing": {
            "ensure_rgb": false,
            "ensure_grayscale": false,
            "imagenet_mode": null,
            "input_scaling": 1.0,
            "pad_to_stride": 1,
            "resize_and_pad_to_target": true,
            "target_height": 1280,
            "target_width": 1280
        },
        "instance_cropping": {
            "center_on_part": "abdomen",
            "crop_size": 256,
            "crop_size_detection_padding": 16
        }
    },
    "model": {
        "backbone": {
            "leap": null,
            "unet": {
                "stem_stride": null,
                "max_stride": 128,
                "output_stride": 4,
                "filters": 24,
                "filters_rate": 2.0,
                "middle_block": true,
                "up_interpolate": true,
                "stacks": 1
            },
            "hourglass": null,
            "resnet": null,
            "pretrained_encoder": null
        },
        "heads": {
            "single_instance": null,
            "centroid": null,
            "centered_instance": {
                "anchor_part": "abdomen",
                "part_names": [
                    "head",
                    "thorax",
                    "abdomen",
                    "wingL",
                    "wingR",
                    "forelegL4",
                    "forelegR4",
                    "midlegL4",
                    "midlegR4",
                    "hindlegL4",
                    "hindlegR4",
                    "eyeL",
                    "eyeR"
                ],
                "sigma": 2.5,
                "output_stride": 4,
                "loss_weight": 1.0,
                "offset_refinement": false
            },
            "multi_instance": null,
            "multi_class_bottomup": null,
            "multi_class_topdown": null
        },
        "base_checkpoint": null
    },
    "optimization": {
        "preload_data": true,
        "augmentation_config": {
            "rotate": true,
            "rotation_min_angle": -180.0,
            "rotation_max_angle": 180.0,
            "translate": false,
            "translate_min": -5,
            "translate_max": 5,
            "scale": false,
            "scale_min": 0.9,
            "scale_max": 1.1,
            "uniform_noise": false,
            "uniform_noise_min_val": 0.0,
            "uniform_noise_max_val": 10.0,
            "gaussian_noise": false,
            "gaussian_noise_mean": 5.0,
            "gaussian_noise_stddev": 1.0,
            "contrast": false,
            "contrast_min_gamma": 0.5,
            "contrast_max_gamma": 2.0,
            "brightness": false,
            "brightness_min_val": 0.0,
            "brightness_max_val": 10.0,
            "random_crop": false,
            "random_crop_height": 256,
            "random_crop_width": 256,
            "random_flip": false,
            "flip_horizontal": false
        },
        "online_shuffling": true,
        "shuffle_buffer_size": 128,
        "prefetch": true,
        "batch_size": 4,
        "batches_per_epoch": 200,
        "min_batches_per_epoch": 200,
        "val_batches_per_epoch": 10,
        "min_val_batches_per_epoch": 10,
        "epochs": 2,
        "optimizer": "adam",
        "initial_learning_rate": 0.0001,
        "learning_rate_schedule": {
            "reduce_on_plateau": true,
            "reduction_factor": 0.5,
            "plateau_min_delta": 1e-06,
            "plateau_patience": 5,
            "plateau_cooldown": 3,
            "min_learning_rate": 1e-08
        },
        "hard_keypoint_mining": {
            "online_mining": false,
            "hard_to_easy_ratio": 2.0,
            "min_hard_keypoints": 2,
            "max_hard_keypoints": null,
            "loss_scale": 5.0
        },
        "early_stopping": {
            "stop_training_on_plateau": true,
            "plateau_min_delta": 1e-08,
            "plateau_patience": 10
        }
    },
    "outputs": {
        "save_outputs": true,
        "run_name": "241009_114536.centered_instance.n=3",
        "run_name_prefix": "",
        "run_name_suffix": "",
        "runs_folder": "D:/social-leap-estimates-animal-poses/datasets/drosophila-melanogaster-courtship/drosophila-melanogaster-courtship\\models",
        "tags": [
            ""
        ],
        "save_visualizations": true,
        "keep_viz_images": false,
        "zip_outputs": false,
        "log_to_csv": true,
        "checkpointing": {
            "initial_model": false,
            "best_model": true,
            "every_epoch": false,
            "latest_model": false,
            "final_model": false
        },
        "tensorboard": {
            "write_logs": false,
            "loss_frequency": "epoch",
            "architecture_graph": false,
            "profile_graph": false,
            "visualizations": true
        },
        "zmq": {
            "subscribe_to_controller": true,
            "controller_address": "tcp://127.0.0.1:8998",
            "controller_polling_timeout": 10,
            "publish_updates": true,
            "publish_address": "tcp://127.0.0.1:9001"
        }
    },
    "name": "",
    "description": "",
    "sleap_version": "1.4.1a2",
    "filename": "C:\\Users\\TalmoLab\\AppData\\Local\\Temp\\tmp1y6bl0lp\\241009_114536_training_job.json"
}
INFO:sleap.nn.training:
INFO:sleap.nn.training:Auto-selected GPU 0 with 23459 MiB of free memory.
INFO:sleap.nn.training:Using GPU 0 for acceleration.
INFO:sleap.nn.training:Disabled GPU memory pre-allocation.
INFO:sleap.nn.training:System:
GPUs: 1/1 available
  Device: /physical_device:GPU:0
         Available: True
       Initialized: False
     Memory growth: True
INFO:sleap.nn.training:
INFO:sleap.nn.training:Initializing trainer...
INFO:sleap.nn.training:Loading training labels from: D:/social-leap-estimates-animal-poses/datasets/drosophila-melanogaster-courtship/drosophila-melanogaster-courtship/labels.v001.slp
INFO:sleap.nn.training:Creating training and validation splits from validation fraction: 0.1
INFO:sleap.nn.training:  Splits: Training = 2 / Validation = 1.
INFO:sleap.nn.training:Setting up for training...
INFO:sleap.nn.training:Setting up pipeline builders...
INFO:sleap.nn.training:Setting up model...
INFO:sleap.nn.training:Building test pipeline...
2024-10-09 11:45:42.918245: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX AVX2
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
2024-10-09 11:45:43.267306: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1532] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 21631 MB memory:  -> device: 0, name: NVIDIA RTX A5000, pci bus id: 0000:01:00.0, compute capability: 8.6
2024-10-09 11:45:45.138164: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1280 } dim { size: 1280 } dim { size: 1 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "103" frequency: 3600 num_cores: 16 environment { key: "cpu_instruction_set" value: "SSE, SSE2" } environment { key: "eigen" value: "3.4.90" } l1_cache_size: 49152 l2_cache_size: 524288 l3_cache_size: 16777216 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 256 } dim { size: 256 } dim { size: 1 } } }
INFO:sleap.nn.training:Loaded test example. [2.358s]
INFO:sleap.nn.training:  Input shape: (256, 256, 1)
INFO:sleap.nn.training:Created Keras model.
INFO:sleap.nn.training:  Backbone: UNet(stacks=1, filters=24, filters_rate=2.0, kernel_size=3, stem_kernel_size=7, convs_per_block=2, stem_blocks=0, down_blocks=7, middle_block=True, up_blocks=5, up_interpolate=True, block_contraction=False)
INFO:sleap.nn.training:  Max stride: 128
INFO:sleap.nn.training:  Parameters: 283,019,413
INFO:sleap.nn.training:  Heads:
INFO:sleap.nn.training:    [0] = CenteredInstanceConfmapsHead(part_names=['head', 'thorax', 'abdomen', 'wingL', 'wingR', 'forelegL4', 'forelegR4', 'midlegL4', 'midlegR4', 'hindlegL4', 'hindlegR4', 'eyeL', 'eyeR'], anchor_part='abdomen', sigma=2.5, output_stride=4, loss_weight=1.0)
INFO:sleap.nn.training:  Outputs:
INFO:sleap.nn.training:    [0] = KerasTensor(type_spec=TensorSpec(shape=(None, 64, 64, 13), dtype=tf.float32, name=None), name='CenteredInstanceConfmapsHead/BiasAdd:0', description="created by layer 'CenteredInstanceConfmapsHead'")
INFO:sleap.nn.training:Training from scratch
INFO:sleap.nn.training:Setting up data pipelines...
INFO:sleap.nn.training:Training set: n = 2
INFO:sleap.nn.training:Validation set: n = 1
INFO:sleap.nn.training:Setting up optimization...
INFO:sleap.nn.training:  Learning rate schedule: LearningRateScheduleConfig(reduce_on_plateau=True, reduction_factor=0.5, plateau_min_delta=1e-06, plateau_patience=5, plateau_cooldown=3, min_learning_rate=1e-08)
INFO:sleap.nn.training:  Early stopping: EarlyStoppingConfig(stop_training_on_plateau=True, plateau_min_delta=1e-08, plateau_patience=10)
INFO:sleap.nn.training:Setting up outputs...
INFO:sleap.nn.callbacks:Training controller subscribed to: tcp://127.0.0.1:8998 (topic: )
INFO:sleap.nn.training:  ZMQ controller subcribed to: tcp://127.0.0.1:8998
INFO:sleap.nn.callbacks:Progress reporter publishing on: tcp://127.0.0.1:9001 for: not_set
INFO:sleap.nn.training:  ZMQ progress reporter publish on: tcp://127.0.0.1:9001
INFO:sleap.nn.training:Created run path: D:/social-leap-estimates-animal-poses/datasets/drosophila-melanogaster-courtship/drosophila-melanogaster-courtship\models\241009_114536.centered_instance.n=3
INFO:sleap.nn.training:Setting up visualization...
2024-10-09 11:45:46.311081: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1280 } dim { size: 1280 } dim { size: 1 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "103" frequency: 3600 num_cores: 16 environment { key: "cpu_instruction_set" value: "SSE, SSE2" } environment { key: "eigen" value: "3.4.90" } l1_cache_size: 49152 l2_cache_size: 524288 l3_cache_size: 16777216 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 256 } dim { size: 256 } dim { size: 1 } } }
2024-10-09 11:45:47.120562: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1280 } dim { size: 1280 } dim { size: 1 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "103" frequency: 3600 num_cores: 16 environment { key: "cpu_instruction_set" value: "SSE, SSE2" } environment { key: "eigen" value: "3.4.90" } l1_cache_size: 49152 l2_cache_size: 524288 l3_cache_size: 16777216 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 256 } dim { size: 256 } dim { size: 1 } } }
INFO:sleap.nn.training:Finished trainer set up. [4.3s]
INFO:sleap.nn.training:Creating tf.data.Datasets for training data generation...
2024-10-09 11:45:48.033762: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1280 } dim { size: 1280 } dim { size: 1 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "103" frequency: 3600 num_cores: 16 environment { key: "cpu_instruction_set" value: "SSE, SSE2" } environment { key: "eigen" value: "3.4.90" } l1_cache_size: 49152 l2_cache_size: 524288 l3_cache_size: 16777216 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 256 } dim { size: 256 } dim { size: 1 } } }
2024-10-09 11:45:49.597663: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1280 } dim { size: 1280 } dim { size: 1 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "103" frequency: 3600 num_cores: 16 environment { key: "cpu_instruction_set" value: "SSE, SSE2" } environment { key: "eigen" value: "3.4.90" } l1_cache_size: 49152 l2_cache_size: 524288 l3_cache_size: 16777216 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 256 } dim { size: 256 } dim { size: 1 } } }
INFO:sleap.nn.training:Finished creating training datasets. [2.7s]
INFO:sleap.nn.training:Starting training loop...
2024-10-09 11:45:50.090577: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1280 } dim { size: 1280 } dim { size: 1 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "103" frequency: 3600 num_cores: 16 environment { key: "cpu_instruction_set" value: "SSE, SSE2" } environment { key: "eigen" value: "3.4.90" } l1_cache_size: 49152 l2_cache_size: 524288 l3_cache_size: 16777216 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 256 } dim { size: 256 } dim { size: 1 } } }
Epoch 1/2
2024-10-09 11:45:52.328777: I tensorflow/stream_executor/cuda/cuda_dnn.cc:384] Loaded cuDNN version 8201
WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.0130s vs `on_train_batch_end` time: 0.1220s). Check your callbacks.
2024-10-09 11:46:29.998998: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1280 } dim { size: 1280 } dim { size: 1 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "103" frequency: 3600 num_cores: 16 environment { key: "cpu_instruction_set" value: "SSE, SSE2" } environment { key: "eigen" value: "3.4.90" } l1_cache_size: 49152 l2_cache_size: 524288 l3_cache_size: 16777216 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 256 } dim { size: 256 } dim { size: 1 } } }
2024-10-09 11:46:31.824231: I tensorflow/stream_executor/cuda/cuda_blas.cc:1786] TensorFloat-32 will be used for the matrix multiplication. This will only be logged once.
2024-10-09 11:46:32.432660: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 13 } dim { size: 64 } dim { size: 64 } dim { size: 1 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "GPU" vendor: "NVIDIA" model: "NVIDIA RTX A5000" frequency: 1695 num_cores: 64 environment { key: "architecture" value: "8.6" } environment { key: "cuda" value: "11020" } environment { key: "cudnn" value: "8100" } num_registers: 65536 l1_cache_size: 24576 l2_cache_size: 6291456 shared_memory_size_per_multiprocessor: 102400 memory_size: 22681944064 bandwidth: 768096000 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: -5 } dim { size: -6 } dim { size: 1 } } }
200/200 - 48s - loss: 0.0024 - head: 0.0018 - thorax: 0.0021 - abdomen: 0.0027 - wingL: 0.0030 - wingR: 0.0031 - forelegL4: 0.0014 - forelegR4: 9.9911e-05 - midlegL4: 0.0027 - midlegR4: 0.0035 - hindlegL4: 0.0016 - hindlegR4: 0.0034 - eyeL: 0.0028 - eyeR: 0.0027 - val_loss: 0.0027 - val_head: 0.0036 - val_thorax: 0.0019 - val_abdomen: 0.0013 - val_wingL: 0.0028 - val_wingR: 0.0032 - val_forelegL4: 0.0013 - val_forelegR4: 2.5745e-04 - val_midlegL4: 0.0033 - val_midlegR4: 0.0033 - val_hindlegL4: 0.0023 - val_hindlegR4: 0.0037 - val_eyeL: 0.0039 - val_eyeR: 0.0037 - lr: 1.0000e-04 - 48s/epoch - 240ms/step
Epoch 2/2
Polling: D:/social-leap-estimates-animal-poses/datasets/drosophila-melanogaster-courtship/drosophila-melanogaster-courtship\models\241009_114324.centroid.n=3\viz\validation.*.png
Polling: D:/social-leap-estimates-animal-poses/datasets/drosophila-melanogaster-courtship/drosophila-melanogaster-courtship\models\241009_114536.centered_instance.n=3\viz\validation.*.png
2024-10-09 11:47:12.089059: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: 1 } dim { size: 1280 } dim { size: 1280 } dim { size: 1 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "103" frequency: 3600 num_cores: 16 environment { key: "cpu_instruction_set" value: "SSE, SSE2" } environment { key: "eigen" value: "3.4.90" } l1_cache_size: 49152 l2_cache_size: 524288 l3_cache_size: 16777216 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 256 } dim { size: 256 } dim { size: 1 } } }
200/200 - 40s - loss: 8.4480e-04 - head: 6.9070e-04 - thorax: 6.0177e-04 - abdomen: 7.6382e-04 - wingL: 7.1380e-04 - wingR: 7.3655e-04 - forelegL4: 0.0011 - forelegR4: 8.6102e-05 - midlegL4: 0.0012 - midlegR4: 9.9908e-04 - hindlegL4: 0.0013 - hindlegR4: 0.0014 - eyeL: 7.9498e-04 - eyeR: 7.0381e-04 - val_loss: 0.0025 - val_head: 0.0040 - val_thorax: 0.0015 - val_abdomen: 0.0012 - val_wingL: 0.0025 - val_wingR: 0.0025 - val_forelegL4: 0.0012 - val_forelegR4: 9.1834e-05 - val_midlegL4: 0.0028 - val_midlegR4: 0.0033 - val_hindlegL4: 0.0020 - val_hindlegR4: 0.0047 - val_eyeL: 0.0030 - val_eyeR: 0.0030 - lr: 1.0000e-04 - 40s/epoch - 202ms/step
INFO:sleap.nn.training:Finished training loop. [1.5 min]
INFO:sleap.nn.training:Deleting visualization directory: D:/social-leap-estimates-animal-poses/datasets/drosophila-melanogaster-courtship/drosophila-melanogaster-courtship\models\241009_114536.centered_instance.n=3\viz
INFO:sleap.nn.training:Saving evaluation metrics to model folder...
Predicting... ----------------------------------------   0% ETA: -:--:-- ?Polling: D:/social-leap-estimates-animal-poses/datasets/drosophila-melanogaster-courtship/drosophila-melanogaster-courtship\models\241009_114324.centroid.n=3\viz\validation.*.png
Polling: D:/social-leap-estimates-animal-poses/datasets/drosophila-melanogaster-courtship/drosophila-melanogaster-courtship\models\241009_114536.centered_instance.n=3\viz\validation.*.png
2024-10-09 11:47:20.567626: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_UINT8 } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_UINT8 shape { dim { size: 2 } dim { size: 1280 } dim { size: 1280 } dim { size: 1 } } } inputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -2 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "CPU" vendor: "GenuineIntel" model: "103" frequency: 3600 num_cores: 16 environment { key: "cpu_instruction_set" value: "SSE, SSE2" } environment { key: "eigen" value: "3.4.90" } l1_cache_size: 49152 l2_cache_size: 524288 l3_cache_size: 16777216 memory_size: 268435456 } outputs { dtype: DT_FLOAT shape { dim { size: -2 } dim { size: -24 } dim { size: -25 } dim { size: 1 } } }
2024-10-09 11:47:20.579069: W tensorflow/core/grappler/costs/op_level_cost_estimator.cc:690] Error in PredictCost() for the op: op: "CropAndResize" attr { key: "T" value { type: DT_FLOAT } } attr { key: "extrapolation_value" value { f: 0 } } attr { key: "method" value { s: "bilinear" } } inputs { dtype: DT_FLOAT shape { dim { size: -65 } dim { size: -66 } dim { size: -67 } dim { size: 1 } } } inputs { dtype: DT_FLOAT shape { dim { size: -9 } dim { size: 4 } } } inputs { dtype: DT_INT32 shape { dim { size: -9 } } } inputs { dtype: DT_INT32 shape { dim { size: 2 } } } device { type: "GPU" vendor: "NVIDIA" model: "NVIDIA RTX A5000" frequency: 1695 num_cores: 64 environment { key: "architecture" value: "8.6" } environment { key: "cuda" value: "11020" } environment { key: "cudnn" value: "8100" } num_registers: 65536 l1_cache_size: 24576 l2_cache_size: 6291456 shared_memory_size_per_multiprocessor: 102400 memory_size: 22681944064 bandwidth: 768096000 } outputs { dtype: DT_FLOAT shape { dim { size: -9 } dim { size: -70 } dim { size: -71 } dim { size: 1 } } }
error: Can't find libdevice directory ${CUDA_DIR}/nvvm/libdevice
error: Can't find libdevice directory ${CUDA_DIR}/nvvm/libdevice
error: Can't find libdevice directory ${CUDA_DIR}/nvvm/libdevice
error: Can't find libdevice directory ${CUDA_DIR}/nvvm/libdevice
error: Can't find libdevice directory ${CUDA_DIR}/nvvm/libdevice
error: Can't find libdevice directory ${CUDA_DIR}/nvvm/libdevice
error: Can't find libdevice directory ${CUDA_DIR}/nvvm/libdevice
error: Can't find libdevice directory ${CUDA_DIR}/nvvm/libdevice
error: Can't find libdevice directory ${CUDA_DIR}/nvvm/libdevice
error: Can't find libdevice directory ${CUDA_DIR}/nvvm/libdevice
error: Can't find libdevice directory ${CUDA_DIR}/nvvm/libdevice
error: Can't find libdevice directory ${CUDA_DIR}/nvvm/libdevice
error: Can't find libdevice directory ${CUDA_DIR}/nvvm/libdevice
2024-10-09 11:47:21.848472: W tensorflow/core/framework/op_kernel.cc:1733] UNKNOWN: JIT compilation failed.
Predicting... ----------------------------------------   0% ETA: -:--:-- ?
Traceback (most recent call last):
  File "\\?\C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\Scripts\sleap-train-script.py", line 33, in <module>
    sys.exit(load_entry_point('sleap==1.4.1a3', 'console_scripts', 'sleap-train')())
  File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\sleap\nn\training.py", line 2039, in main
    trainer.train()
  File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\sleap\nn\training.py", line 953, in train
    self.evaluate()
  File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\sleap\nn\training.py", line 961, in evaluate
    sleap.nn.evals.evaluate_model(
  File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\sleap\nn\evals.py", line 744, in evaluate_model
    labels_pr: Labels = predictor.predict(labels_gt, make_labels=True)
  File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\sleap\nn\inference.py", line 527, in predict
    self._make_labeled_frames_from_generator(generator, data)
  File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\sleap\nn\inference.py", line 2645, in _make_labeled_frames_from_generator
    for ex in generator:
  File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\sleap\nn\inference.py", line 437, in _predict_generator
    ex = process_batch(ex)
  File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\sleap\nn\inference.py", line 400, in process_batch
    preds = self.inference_model.predict_on_batch(ex, numpy=True)
  File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\sleap\nn\inference.py", line 1070, in predict_on_batch
    outs = super().predict_on_batch(data, **kwargs)
  File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\keras\engine\training.py", line 2230, in predict_on_batch
    outputs = self.predict_function(iterator)
  File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\tensorflow\python\util\traceback_utils.py", line 153, in error_handler
    raise e.with_traceback(filtered_tb) from None
  File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\tensorflow\python\eager\execute.py", line 54, in quick_execute
    tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,
tensorflow.python.framework.errors_impl.UnknownError: Graph execution error:

Detected at node 'FloorMod' defined at (most recent call last):
    File "\\?\C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\Scripts\sleap-train-script.py", line 33, in <module>
      sys.exit(load_entry_point('sleap==1.4.1a3', 'console_scripts', 'sleap-train')())
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\sleap\nn\training.py", line 2039, in main
      trainer.train()
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\sleap\nn\training.py", line 953, in train
      self.evaluate()
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\sleap\nn\training.py", line 961, in evaluate
      sleap.nn.evals.evaluate_model(
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\sleap\nn\evals.py", line 744, in evaluate_model
      labels_pr: Labels = predictor.predict(labels_gt, make_labels=True)
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\sleap\nn\inference.py", line 527, in predict
      self._make_labeled_frames_from_generator(generator, data)
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\sleap\nn\inference.py", line 2645, in _make_labeled_frames_from_generator
      for ex in generator:
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\sleap\nn\inference.py", line 437, in _predict_generator
      ex = process_batch(ex)
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\sleap\nn\inference.py", line 400, in process_batch
      preds = self.inference_model.predict_on_batch(ex, numpy=True)
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\sleap\nn\inference.py", line 1070, in predict_on_batch
      outs = super().predict_on_batch(data, **kwargs)
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\keras\engine\training.py", line 2230, in predict_on_batch
      outputs = self.predict_function(iterator)
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\keras\engine\training.py", line 1845, in predict_function
      return step_function(self, iterator)
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\keras\engine\training.py", line 1834, in step_function
      outputs = model.distribute_strategy.run(run_step, args=(data,))
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\keras\engine\training.py", line 1823, in run_step
      outputs = model.predict_step(data)
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\keras\engine\training.py", line 1791, in predict_step
      return self(x, training=False)
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\keras\utils\traceback_utils.py", line 64, in error_handler
      return fn(*args, **kwargs)
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\keras\engine\training.py", line 490, in __call__
      return super().__call__(*args, **kwargs)
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\keras\utils\traceback_utils.py", line 64, in error_handler
      return fn(*args, **kwargs)
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\keras\engine\base_layer.py", line 1014, in __call__
      outputs = call_fn(inputs, *args, **kwargs)
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\keras\utils\traceback_utils.py", line 92, in error_handler
      return fn(*args, **kwargs)
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\sleap\nn\inference.py", line 2267, in call
      if isinstance(self.instance_peaks, FindInstancePeaksGroundTruth):
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\sleap\nn\inference.py", line 2276, in call
      peaks_output = self.instance_peaks(crop_output)
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\keras\utils\traceback_utils.py", line 64, in error_handler
      return fn(*args, **kwargs)
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\keras\engine\base_layer.py", line 1014, in __call__
      outputs = call_fn(inputs, *args, **kwargs)
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\keras\utils\traceback_utils.py", line 92, in error_handler
      return fn(*args, **kwargs)
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\sleap\nn\inference.py", line 2112, in call
      if self.offsets_ind is None:
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\sleap\nn\inference.py", line 2114, in call
      peak_points, peak_vals = sleap.nn.peak_finding.find_global_peaks(
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\sleap\nn\peak_finding.py", line 366, in find_global_peaks
      rough_peaks, peak_vals = find_global_peaks_rough(
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\sleap\nn\peak_finding.py", line 224, in find_global_peaks_rough
      channel_subs = tf.math.mod(tf.range(total_peaks, dtype=tf.int64), channels)
Node: 'FloorMod'
Detected at node 'FloorMod' defined at (most recent call last):
    File "\\?\C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\Scripts\sleap-train-script.py", line 33, in <module>
      sys.exit(load_entry_point('sleap==1.4.1a3', 'console_scripts', 'sleap-train')())
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\sleap\nn\training.py", line 2039, in main
      trainer.train()
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\sleap\nn\training.py", line 953, in train
      self.evaluate()
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\sleap\nn\training.py", line 961, in evaluate
      sleap.nn.evals.evaluate_model(
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\sleap\nn\evals.py", line 744, in evaluate_model
      labels_pr: Labels = predictor.predict(labels_gt, make_labels=True)
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\sleap\nn\inference.py", line 527, in predict
      self._make_labeled_frames_from_generator(generator, data)
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\sleap\nn\inference.py", line 2645, in _make_labeled_frames_from_generator
      for ex in generator:
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\sleap\nn\inference.py", line 437, in _predict_generator
      ex = process_batch(ex)
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\sleap\nn\inference.py", line 400, in process_batch
      preds = self.inference_model.predict_on_batch(ex, numpy=True)
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\sleap\nn\inference.py", line 1070, in predict_on_batch
      outs = super().predict_on_batch(data, **kwargs)
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\keras\engine\training.py", line 2230, in predict_on_batch
      outputs = self.predict_function(iterator)
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\keras\engine\training.py", line 1845, in predict_function
      return step_function(self, iterator)
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\keras\engine\training.py", line 1834, in step_function
      outputs = model.distribute_strategy.run(run_step, args=(data,))
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\keras\engine\training.py", line 1823, in run_step
      outputs = model.predict_step(data)
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\keras\engine\training.py", line 1791, in predict_step
      return self(x, training=False)
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\keras\utils\traceback_utils.py", line 64, in error_handler
      return fn(*args, **kwargs)
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\keras\engine\training.py", line 490, in __call__
      return super().__call__(*args, **kwargs)
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\keras\utils\traceback_utils.py", line 64, in error_handler
      return fn(*args, **kwargs)
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\keras\engine\base_layer.py", line 1014, in __call__
      outputs = call_fn(inputs, *args, **kwargs)
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\keras\utils\traceback_utils.py", line 92, in error_handler
      return fn(*args, **kwargs)
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\sleap\nn\inference.py", line 2267, in call
      if isinstance(self.instance_peaks, FindInstancePeaksGroundTruth):
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\sleap\nn\inference.py", line 2276, in call
      peaks_output = self.instance_peaks(crop_output)
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\keras\utils\traceback_utils.py", line 64, in error_handler
      return fn(*args, **kwargs)
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\keras\engine\base_layer.py", line 1014, in __call__
      outputs = call_fn(inputs, *args, **kwargs)
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\keras\utils\traceback_utils.py", line 92, in error_handler
      return fn(*args, **kwargs)
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\sleap\nn\inference.py", line 2112, in call
      if self.offsets_ind is None:
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\sleap\nn\inference.py", line 2114, in call
      peak_points, peak_vals = sleap.nn.peak_finding.find_global_peaks(
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\sleap\nn\peak_finding.py", line 366, in find_global_peaks
      rough_peaks, peak_vals = find_global_peaks_rough(
    File "C:\Users\TalmoLab\mambaforge\envs\sleap_1.4.1a3_py310\lib\site-packages\sleap\nn\peak_finding.py", line 224, in find_global_peaks_rough
      channel_subs = tf.math.mod(tf.range(total_peaks, dtype=tf.int64), channels)
Node: 'FloorMod'
2 root error(s) found.
  (0) UNKNOWN:  JIT compilation failed.
         [[{{node FloorMod}}]]
         [[top_down_inference_model/find_instance_peaks_1/RaggedFromValueRowIds_1/RowPartitionFromValueRowIds/assert_less/Assert/AssertGuard/pivot_f/_177/_415]]
  (1) UNKNOWN:  JIT compilation failed.
         [[{{node FloorMod}}]]
0 successful operations.
0 derived errors ignored. [Op:__inference_predict_function_31992]
INFO:sleap.nn.callbacks:Closing the reporter controller/context.
INFO:sleap.nn.callbacks:Closing the training controller socket/context.
Run Path: D:/social-leap-estimates-animal-poses/datasets/drosophila-melanogaster-courtship/drosophila-melanogaster-courtship\models\241009_114536.centered_instance.n=3